{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D2H6YLCPOXvZ",
        "outputId": "5bfb0564-b903-453b-fdb5-80a7186c2c9c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.23.1-py3-none-any.whl (5.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.3 MB 4.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 70.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Collecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",
            "\u001b[K     |████████████████████████████████| 163 kB 83.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.9.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.1 transformers-4.23.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.6.1-py3-none-any.whl (441 kB)\n",
            "\u001b[K     |████████████████████████████████| 441 kB 4.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 75.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.8.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.10.1)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.13.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.1)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.13-py37-none-any.whl (115 kB)\n",
            "\u001b[K     |████████████████████████████████| 115 kB 78.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.1.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.8.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 80.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.9.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: urllib3, xxhash, responses, multiprocess, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.6.1 multiprocess-0.70.13 responses-0.18.0 urllib3-1.25.11 xxhash-3.1.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (1.21.6)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.3.5)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.21.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2022.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=18oZZ4jqRK-uF-Nz6ftRdgNjKix88hrnO\n",
            "To: /content/data_and_models.zip\n",
            "100% 33.3M/33.3M [00:00<00:00, 113MB/s]\n",
            "Archive:  data_and_models.zip\n",
            "   creating: data_and_models/\n",
            "  inflating: __MACOSX/._data_and_models  \n",
            "  inflating: data_and_models/logistic_model_8.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._logistic_model_8.pkl  \n",
            "  inflating: data_and_models/tfidf_44.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._tfidf_44.pkl  \n",
            "  inflating: data_and_models/tfidf_8.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._tfidf_8.pkl  \n",
            "  inflating: data_and_models/target_corpus.csv  \n",
            "  inflating: __MACOSX/data_and_models/._target_corpus.csv  \n",
            "  inflating: data_and_models/logistic_model_44.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._logistic_model_44.pkl  \n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install datasets\n",
        "!pip install --upgrade --no-cache-dir gdown==4.5.4\n",
        "\n",
        "!gdown 18oZZ4jqRK-uF-Nz6ftRdgNjKix88hrnO\n",
        "!unzip data_and_models.zip && rm data_and_models.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "8ece54a75604436f9a36b5ccdeca7d11",
            "e85e18974a55458fafe8cea394dda2d1",
            "3652b1df191d4326814059acadbf24c6",
            "2bf51e9622ed462d8b55a45f12f2e915",
            "d3c5bf2b63f143eda03ebbacf1042c4d",
            "7145848d480d4740a448dbba969454b1",
            "cbf68140c81c42cca33439cd062d4385",
            "a389a343a98b47669bde848a34adfab6",
            "25ff7e39e5ec488f8bbad2168fba4a44",
            "c3561d189a4e461792d395f933521ddb",
            "187b2c658c754f25be271e0e27774149",
            "b308c42532cb4491ba78546a312b347e",
            "45534ee3df6e483287993565172cdca0",
            "aee36f2e6cc84db2b8c587c2cf07094e",
            "dcb07a89a02e41beb452fe3562b8aa31",
            "4693a236332c420e8b0d847a086c6a99",
            "2d1bc5b4baea44da8b83e9f9ff24be82",
            "428a0c9f254744f18865480449e440da",
            "b457c96412c94a4399e0a9bfc60e135f",
            "6e4a736d0dda4c0ba86ed4adce30bf06",
            "c99f55fd2ff246e19e75ea8a3eea8ea2",
            "3cba08ef6faa41e89534c785cd157259",
            "ebc04124b24c4647af5a43e6055c1363",
            "6fbde38980d84dd281fdb780254018aa",
            "3cae7a02f919431a9e74509321c27783",
            "fdbf832dc70d4a64be71dfcbfd9ce0ee",
            "ce4f4f8a574a4c8c86c104b11cbf23ce",
            "8dfb8ee3173841f69dc9cbb5507504e8",
            "2a9f078224974a1eaf9d16c93ba627fe",
            "040a69ae25d847d68fb4b6d37a3c1f7f",
            "51c190453a6e49f8ae3d5a479f949ef6",
            "6f2bc607548944e79bea5dfd360c5d59",
            "b7716b58a6054808967e269832b8be96",
            "ce50c195a786425fa88813c88810ffba",
            "bdf3a830b2824792b3cbb4731e68fdf1",
            "389fb966970a45c68d58d76a927834ee",
            "59c21d08241f4546b1458534d2c65137",
            "e051bd21f5a942139bfd9fd788720c6d",
            "e3965d9988284cae8e4c5a31e315fdd4",
            "2999cd51404e4bd8b40fa2c45b381bb8",
            "dc46a710d3e14349adcad4b1c3d74891",
            "099cc60e0e2340ac881c6beb35594b64",
            "5ab058186b5a47a08a4d2eb3d4ddad40",
            "23a003f440b545359b8010e9a7083e03",
            "238665a8a4cc45aa9b2d3d85380b5d4f",
            "acd03816260e4583bbaf11c1fcd56982",
            "2016118b6be84040af0d2c0586c43d14",
            "417c0e82d111436fac1232d4a7850d64",
            "c36f73c489cc430aa631af2f1c010d59",
            "9e0d568b2f594f3b9dfb6641797716a9",
            "f4b87dde40474fb792832cb12deea15f",
            "dcfc958e8b4146a3a41708180c72cee5",
            "e64c1c8790f549eea818bdb50a01feb6",
            "e23ecf61ef3a4f71ad51da5ae972d7a4",
            "ba046666f06e4b61a5ea392314078d3e",
            "6e1301421a704289a6bb7394ad3d1bdc",
            "adb090506cbe47b6b39d4cb3888f87f2",
            "ed618e494bdf48a3af03dcb604316003",
            "b4b862be7f244e79a276f0c79d1b848a",
            "60896d137fae47f0bac72919b9735ef0",
            "68ad546783a74e96830270f34303e52a",
            "4a6cd193e4574f3fbddf0efb04e4a933",
            "d9c741d200cf4c89b8fe66055c570a17",
            "d73455019c34462b901793a560df11d7",
            "17ce202f23f7434ebafcdc77cf6d4e00",
            "b560b6682f6442fc8e056587a96047bb"
          ]
        },
        "id": "H-LANn-hUlZh",
        "outputId": "ea8fa77d-d53b-4b4b-c3e2-e47f2de6514e"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8ece54a75604436f9a36b5ccdeca7d11",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/899k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b308c42532cb4491ba78546a312b347e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ebc04124b24c4647af5a43e6055c1363",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ce50c195a786425fa88813c88810ffba",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/481 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "200 1882 1883\n",
            "# classes in train 34\n",
            "# classes in dev 40\n",
            "# classes in test 39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "238665a8a4cc45aa9b2d3d85380b5d4f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/501M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:10, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.637300</td>\n",
              "      <td>3.438698</td>\n",
              "      <td>0.171095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.390800</td>\n",
              "      <td>3.317014</td>\n",
              "      <td>0.171095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.150000</td>\n",
              "      <td>3.221827</td>\n",
              "      <td>0.171095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.953700</td>\n",
              "      <td>3.116266</td>\n",
              "      <td>0.193411</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.762500</td>\n",
              "      <td>2.936279</td>\n",
              "      <td>0.268863</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.434800</td>\n",
              "      <td>2.855510</td>\n",
              "      <td>0.274708</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.892900</td>\n",
              "      <td>2.771087</td>\n",
              "      <td>0.289586</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.812500</td>\n",
              "      <td>2.705738</td>\n",
              "      <td>0.320935</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.469600</td>\n",
              "      <td>2.653631</td>\n",
              "      <td>0.334750</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>1.149400</td>\n",
              "      <td>2.606690</td>\n",
              "      <td>0.357067</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>1.031000</td>\n",
              "      <td>2.585628</td>\n",
              "      <td>0.373539</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.891700</td>\n",
              "      <td>2.584399</td>\n",
              "      <td>0.374070</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.877500</td>\n",
              "      <td>2.565406</td>\n",
              "      <td>0.370351</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.676700</td>\n",
              "      <td>2.557717</td>\n",
              "      <td>0.383103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.558500</td>\n",
              "      <td>2.585047</td>\n",
              "      <td>0.372476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.546000</td>\n",
              "      <td>2.578568</td>\n",
              "      <td>0.383103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.456200</td>\n",
              "      <td>2.576812</td>\n",
              "      <td>0.382040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.410700</td>\n",
              "      <td>2.588730</td>\n",
              "      <td>0.386291</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.437400</td>\n",
              "      <td>2.590993</td>\n",
              "      <td>0.389479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.386600</td>\n",
              "      <td>2.592169</td>\n",
              "      <td>0.385228</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:14: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
            "  \n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6e1301421a704289a6bb7394ad3d1bdc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/1.65k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-247 (score: 0.3894792773645058).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "200 1882 1883\n",
            "# classes in train 31\n",
            "# classes in dev 41\n",
            "# classes in test 40\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:03, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.706800</td>\n",
              "      <td>3.351836</td>\n",
              "      <td>0.196068</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.279700</td>\n",
              "      <td>3.185449</td>\n",
              "      <td>0.196068</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.187600</td>\n",
              "      <td>3.008971</td>\n",
              "      <td>0.259299</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.651400</td>\n",
              "      <td>2.895990</td>\n",
              "      <td>0.265675</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.528800</td>\n",
              "      <td>2.776418</td>\n",
              "      <td>0.285335</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.234400</td>\n",
              "      <td>2.680371</td>\n",
              "      <td>0.328374</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.710100</td>\n",
              "      <td>2.655349</td>\n",
              "      <td>0.315090</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.427900</td>\n",
              "      <td>2.592257</td>\n",
              "      <td>0.340595</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.247700</td>\n",
              "      <td>2.599226</td>\n",
              "      <td>0.337407</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.943600</td>\n",
              "      <td>2.568644</td>\n",
              "      <td>0.362380</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.751900</td>\n",
              "      <td>2.579175</td>\n",
              "      <td>0.345377</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.757300</td>\n",
              "      <td>2.565231</td>\n",
              "      <td>0.360255</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.627200</td>\n",
              "      <td>2.571746</td>\n",
              "      <td>0.359192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.467300</td>\n",
              "      <td>2.585352</td>\n",
              "      <td>0.354942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.398000</td>\n",
              "      <td>2.578546</td>\n",
              "      <td>0.359724</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.386800</td>\n",
              "      <td>2.608116</td>\n",
              "      <td>0.356536</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.318700</td>\n",
              "      <td>2.593635</td>\n",
              "      <td>0.365569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.316500</td>\n",
              "      <td>2.605255</td>\n",
              "      <td>0.357067</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.270200</td>\n",
              "      <td>2.604900</td>\n",
              "      <td>0.366631</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.278800</td>\n",
              "      <td>2.610849</td>\n",
              "      <td>0.367163</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-260 (score: 0.3671625929861849).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "200 1882 1883\n",
            "# classes in train 32\n",
            "# classes in dev 41\n",
            "# classes in test 41\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:00, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.611400</td>\n",
              "      <td>3.352728</td>\n",
              "      <td>0.195537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.256700</td>\n",
              "      <td>3.201875</td>\n",
              "      <td>0.195537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.958900</td>\n",
              "      <td>3.047744</td>\n",
              "      <td>0.238576</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.597400</td>\n",
              "      <td>2.878942</td>\n",
              "      <td>0.293836</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.450400</td>\n",
              "      <td>2.856372</td>\n",
              "      <td>0.307120</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.184200</td>\n",
              "      <td>2.686482</td>\n",
              "      <td>0.353879</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.720300</td>\n",
              "      <td>2.633838</td>\n",
              "      <td>0.358130</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.550000</td>\n",
              "      <td>2.603144</td>\n",
              "      <td>0.383634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.329500</td>\n",
              "      <td>2.581018</td>\n",
              "      <td>0.384166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>1.107200</td>\n",
              "      <td>2.571044</td>\n",
              "      <td>0.391073</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.986900</td>\n",
              "      <td>2.565149</td>\n",
              "      <td>0.389479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.801500</td>\n",
              "      <td>2.535970</td>\n",
              "      <td>0.397981</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.726600</td>\n",
              "      <td>2.555365</td>\n",
              "      <td>0.403826</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.553700</td>\n",
              "      <td>2.550322</td>\n",
              "      <td>0.395855</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.543800</td>\n",
              "      <td>2.583018</td>\n",
              "      <td>0.404357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.505700</td>\n",
              "      <td>2.548708</td>\n",
              "      <td>0.402232</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.455300</td>\n",
              "      <td>2.572143</td>\n",
              "      <td>0.401700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.423800</td>\n",
              "      <td>2.580368</td>\n",
              "      <td>0.401700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.396300</td>\n",
              "      <td>2.576137</td>\n",
              "      <td>0.403826</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.410300</td>\n",
              "      <td>2.582048</td>\n",
              "      <td>0.404357</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-195 (score: 0.40435706695005313).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "200 1882 1883\n",
            "# classes in train 30\n",
            "# classes in dev 39\n",
            "# classes in test 42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:02, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.665600</td>\n",
              "      <td>3.365279</td>\n",
              "      <td>0.184910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.307200</td>\n",
              "      <td>3.252143</td>\n",
              "      <td>0.184910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.176300</td>\n",
              "      <td>3.109071</td>\n",
              "      <td>0.218385</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.617200</td>\n",
              "      <td>2.972684</td>\n",
              "      <td>0.273645</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.406400</td>\n",
              "      <td>2.880045</td>\n",
              "      <td>0.302869</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.274500</td>\n",
              "      <td>2.775948</td>\n",
              "      <td>0.323592</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.632200</td>\n",
              "      <td>2.725397</td>\n",
              "      <td>0.347503</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.386700</td>\n",
              "      <td>2.689137</td>\n",
              "      <td>0.343783</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.339700</td>\n",
              "      <td>2.668491</td>\n",
              "      <td>0.352285</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.920100</td>\n",
              "      <td>2.657280</td>\n",
              "      <td>0.362912</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.791300</td>\n",
              "      <td>2.669959</td>\n",
              "      <td>0.357598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.770400</td>\n",
              "      <td>2.620548</td>\n",
              "      <td>0.366631</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.574500</td>\n",
              "      <td>2.617511</td>\n",
              "      <td>0.376727</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.509400</td>\n",
              "      <td>2.619804</td>\n",
              "      <td>0.380978</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.449100</td>\n",
              "      <td>2.602082</td>\n",
              "      <td>0.378852</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.426700</td>\n",
              "      <td>2.637316</td>\n",
              "      <td>0.383634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.364100</td>\n",
              "      <td>2.635736</td>\n",
              "      <td>0.382040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.341400</td>\n",
              "      <td>2.633663</td>\n",
              "      <td>0.383103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.321700</td>\n",
              "      <td>2.645670</td>\n",
              "      <td>0.383103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.288600</td>\n",
              "      <td>2.645833</td>\n",
              "      <td>0.382572</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-208 (score: 0.383634431455898).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "200 1882 1883\n",
            "# classes in train 31\n",
            "# classes in dev 41\n",
            "# classes in test 39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:03, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.653200</td>\n",
              "      <td>3.348582</td>\n",
              "      <td>0.192880</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.377600</td>\n",
              "      <td>3.241226</td>\n",
              "      <td>0.192880</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.241000</td>\n",
              "      <td>3.196209</td>\n",
              "      <td>0.192880</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>3.045800</td>\n",
              "      <td>3.042512</td>\n",
              "      <td>0.257705</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.816900</td>\n",
              "      <td>2.890058</td>\n",
              "      <td>0.279490</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.473000</td>\n",
              "      <td>2.758230</td>\n",
              "      <td>0.307120</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>2.081200</td>\n",
              "      <td>2.704084</td>\n",
              "      <td>0.336876</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.707100</td>\n",
              "      <td>2.619623</td>\n",
              "      <td>0.363974</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.569500</td>\n",
              "      <td>2.577912</td>\n",
              "      <td>0.375133</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>1.123400</td>\n",
              "      <td>2.528905</td>\n",
              "      <td>0.393730</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.993500</td>\n",
              "      <td>2.520982</td>\n",
              "      <td>0.396387</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.851500</td>\n",
              "      <td>2.481378</td>\n",
              "      <td>0.404357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.700200</td>\n",
              "      <td>2.506797</td>\n",
              "      <td>0.402763</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.612100</td>\n",
              "      <td>2.496136</td>\n",
              "      <td>0.408608</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.547100</td>\n",
              "      <td>2.491879</td>\n",
              "      <td>0.401169</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.502800</td>\n",
              "      <td>2.496597</td>\n",
              "      <td>0.404888</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.424500</td>\n",
              "      <td>2.500280</td>\n",
              "      <td>0.403294</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.376300</td>\n",
              "      <td>2.504919</td>\n",
              "      <td>0.403826</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.366400</td>\n",
              "      <td>2.498834</td>\n",
              "      <td>0.410733</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.395200</td>\n",
              "      <td>2.497773</td>\n",
              "      <td>0.413390</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-260 (score: 0.41339001062699254).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "300 1882 1883\n",
            "# classes in train 34\n",
            "# classes in dev 40\n",
            "# classes in test 39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:50, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.684600</td>\n",
              "      <td>3.355794</td>\n",
              "      <td>0.171095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.172300</td>\n",
              "      <td>3.175599</td>\n",
              "      <td>0.171095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.003000</td>\n",
              "      <td>3.014982</td>\n",
              "      <td>0.238576</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.726900</td>\n",
              "      <td>2.783000</td>\n",
              "      <td>0.309245</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.200200</td>\n",
              "      <td>2.588919</td>\n",
              "      <td>0.353879</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.707400</td>\n",
              "      <td>2.514342</td>\n",
              "      <td>0.373007</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.445200</td>\n",
              "      <td>2.454774</td>\n",
              "      <td>0.397450</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.075300</td>\n",
              "      <td>2.441601</td>\n",
              "      <td>0.413921</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.910700</td>\n",
              "      <td>2.427202</td>\n",
              "      <td>0.419766</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.664100</td>\n",
              "      <td>2.426107</td>\n",
              "      <td>0.424017</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.594600</td>\n",
              "      <td>2.457294</td>\n",
              "      <td>0.426142</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.485900</td>\n",
              "      <td>2.506059</td>\n",
              "      <td>0.427736</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.365800</td>\n",
              "      <td>2.471280</td>\n",
              "      <td>0.430393</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.321600</td>\n",
              "      <td>2.510057</td>\n",
              "      <td>0.431456</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.261400</td>\n",
              "      <td>2.540025</td>\n",
              "      <td>0.423486</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.206000</td>\n",
              "      <td>2.582448</td>\n",
              "      <td>0.419766</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.194900</td>\n",
              "      <td>2.599904</td>\n",
              "      <td>0.424548</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.184800</td>\n",
              "      <td>2.594532</td>\n",
              "      <td>0.424017</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.124000</td>\n",
              "      <td>2.606201</td>\n",
              "      <td>0.426142</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.133100</td>\n",
              "      <td>2.612164</td>\n",
              "      <td>0.425080</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-266 (score: 0.4314558979808714).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "300 1882 1883\n",
            "# classes in train 36\n",
            "# classes in dev 41\n",
            "# classes in test 40\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:51, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.665300</td>\n",
              "      <td>3.274939</td>\n",
              "      <td>0.196068</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.246000</td>\n",
              "      <td>3.123399</td>\n",
              "      <td>0.196068</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.995700</td>\n",
              "      <td>2.894967</td>\n",
              "      <td>0.246015</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.634400</td>\n",
              "      <td>2.730283</td>\n",
              "      <td>0.297024</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.094500</td>\n",
              "      <td>2.638974</td>\n",
              "      <td>0.343252</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.737500</td>\n",
              "      <td>2.544092</td>\n",
              "      <td>0.376727</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.528400</td>\n",
              "      <td>2.482233</td>\n",
              "      <td>0.385228</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.089800</td>\n",
              "      <td>2.483757</td>\n",
              "      <td>0.378321</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.872100</td>\n",
              "      <td>2.466243</td>\n",
              "      <td>0.392667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.736900</td>\n",
              "      <td>2.483630</td>\n",
              "      <td>0.384166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.589100</td>\n",
              "      <td>2.528759</td>\n",
              "      <td>0.380978</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.440400</td>\n",
              "      <td>2.508062</td>\n",
              "      <td>0.387354</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.347000</td>\n",
              "      <td>2.528600</td>\n",
              "      <td>0.382572</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.285900</td>\n",
              "      <td>2.556266</td>\n",
              "      <td>0.384166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.217700</td>\n",
              "      <td>2.628875</td>\n",
              "      <td>0.385760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.220500</td>\n",
              "      <td>2.587674</td>\n",
              "      <td>0.392667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.187400</td>\n",
              "      <td>2.613252</td>\n",
              "      <td>0.396918</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.170400</td>\n",
              "      <td>2.630033</td>\n",
              "      <td>0.392136</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.162700</td>\n",
              "      <td>2.631152</td>\n",
              "      <td>0.385760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.156100</td>\n",
              "      <td>2.628581</td>\n",
              "      <td>0.387885</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-323 (score: 0.3969181721572795).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "300 1882 1883\n",
            "# classes in train 32\n",
            "# classes in dev 41\n",
            "# classes in test 41\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:54, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.683500</td>\n",
              "      <td>3.269358</td>\n",
              "      <td>0.195537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.130500</td>\n",
              "      <td>3.176287</td>\n",
              "      <td>0.195537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.022100</td>\n",
              "      <td>2.978158</td>\n",
              "      <td>0.265143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.714600</td>\n",
              "      <td>2.792143</td>\n",
              "      <td>0.301807</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.310000</td>\n",
              "      <td>2.657253</td>\n",
              "      <td>0.351753</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.014800</td>\n",
              "      <td>2.565810</td>\n",
              "      <td>0.359724</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.505900</td>\n",
              "      <td>2.527896</td>\n",
              "      <td>0.390542</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.258100</td>\n",
              "      <td>2.423936</td>\n",
              "      <td>0.418704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.077800</td>\n",
              "      <td>2.381187</td>\n",
              "      <td>0.439426</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.779900</td>\n",
              "      <td>2.362800</td>\n",
              "      <td>0.435175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.733800</td>\n",
              "      <td>2.400760</td>\n",
              "      <td>0.433581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.554500</td>\n",
              "      <td>2.429946</td>\n",
              "      <td>0.424548</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.435200</td>\n",
              "      <td>2.402588</td>\n",
              "      <td>0.430925</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.358000</td>\n",
              "      <td>2.412553</td>\n",
              "      <td>0.433050</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.296700</td>\n",
              "      <td>2.454029</td>\n",
              "      <td>0.439426</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.272000</td>\n",
              "      <td>2.461140</td>\n",
              "      <td>0.435175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.229500</td>\n",
              "      <td>2.472147</td>\n",
              "      <td>0.433581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.177500</td>\n",
              "      <td>2.506775</td>\n",
              "      <td>0.437832</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.161900</td>\n",
              "      <td>2.502243</td>\n",
              "      <td>0.428799</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.181300</td>\n",
              "      <td>2.511707</td>\n",
              "      <td>0.432519</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-171 (score: 0.4394261424017003).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "300 1882 1883\n",
            "# classes in train 33\n",
            "# classes in dev 39\n",
            "# classes in test 42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:51, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.621900</td>\n",
              "      <td>3.290799</td>\n",
              "      <td>0.184910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.077400</td>\n",
              "      <td>3.151619</td>\n",
              "      <td>0.190223</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.852000</td>\n",
              "      <td>3.016177</td>\n",
              "      <td>0.227418</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.511200</td>\n",
              "      <td>2.914029</td>\n",
              "      <td>0.238045</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.169400</td>\n",
              "      <td>2.829404</td>\n",
              "      <td>0.294368</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.806500</td>\n",
              "      <td>2.652848</td>\n",
              "      <td>0.342721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.469400</td>\n",
              "      <td>2.601874</td>\n",
              "      <td>0.353348</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.167200</td>\n",
              "      <td>2.568219</td>\n",
              "      <td>0.387885</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.924200</td>\n",
              "      <td>2.526570</td>\n",
              "      <td>0.400106</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.678900</td>\n",
              "      <td>2.516763</td>\n",
              "      <td>0.383103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.649200</td>\n",
              "      <td>2.545969</td>\n",
              "      <td>0.388948</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.466500</td>\n",
              "      <td>2.559998</td>\n",
              "      <td>0.384166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.376200</td>\n",
              "      <td>2.542032</td>\n",
              "      <td>0.392136</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.291100</td>\n",
              "      <td>2.574109</td>\n",
              "      <td>0.401169</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.284100</td>\n",
              "      <td>2.610612</td>\n",
              "      <td>0.394793</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.217900</td>\n",
              "      <td>2.613878</td>\n",
              "      <td>0.400638</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.198800</td>\n",
              "      <td>2.650246</td>\n",
              "      <td>0.401169</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.175100</td>\n",
              "      <td>2.652514</td>\n",
              "      <td>0.399044</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.179200</td>\n",
              "      <td>2.665014</td>\n",
              "      <td>0.400106</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.159500</td>\n",
              "      <td>2.672340</td>\n",
              "      <td>0.401700</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-380 (score: 0.40170031880977686).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "300 1882 1883\n",
            "# classes in train 34\n",
            "# classes in dev 41\n",
            "# classes in test 39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:53, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.681100</td>\n",
              "      <td>3.248961</td>\n",
              "      <td>0.192880</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.156600</td>\n",
              "      <td>3.065289</td>\n",
              "      <td>0.239639</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.903500</td>\n",
              "      <td>2.817010</td>\n",
              "      <td>0.273114</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.433300</td>\n",
              "      <td>2.633851</td>\n",
              "      <td>0.377258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.964300</td>\n",
              "      <td>2.491365</td>\n",
              "      <td>0.398512</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.487400</td>\n",
              "      <td>2.476863</td>\n",
              "      <td>0.396918</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.211900</td>\n",
              "      <td>2.467585</td>\n",
              "      <td>0.404888</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.923800</td>\n",
              "      <td>2.407140</td>\n",
              "      <td>0.429330</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.796600</td>\n",
              "      <td>2.459828</td>\n",
              "      <td>0.411265</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.512000</td>\n",
              "      <td>2.471809</td>\n",
              "      <td>0.410202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.464500</td>\n",
              "      <td>2.452638</td>\n",
              "      <td>0.418704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.403700</td>\n",
              "      <td>2.465543</td>\n",
              "      <td>0.417641</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.327600</td>\n",
              "      <td>2.450375</td>\n",
              "      <td>0.428799</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.300900</td>\n",
              "      <td>2.519900</td>\n",
              "      <td>0.420829</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.209800</td>\n",
              "      <td>2.549449</td>\n",
              "      <td>0.419235</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.200100</td>\n",
              "      <td>2.559360</td>\n",
              "      <td>0.425611</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.172500</td>\n",
              "      <td>2.585244</td>\n",
              "      <td>0.414453</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.162900</td>\n",
              "      <td>2.590216</td>\n",
              "      <td>0.420298</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.135900</td>\n",
              "      <td>2.596475</td>\n",
              "      <td>0.418172</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.137100</td>\n",
              "      <td>2.609064</td>\n",
              "      <td>0.417109</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-152 (score: 0.4293304994686504).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "400 1882 1883\n",
            "# classes in train 36\n",
            "# classes in dev 40\n",
            "# classes in test 39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:40, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.349700</td>\n",
              "      <td>3.290063</td>\n",
              "      <td>0.171095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.095200</td>\n",
              "      <td>3.141652</td>\n",
              "      <td>0.175345</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.785000</td>\n",
              "      <td>2.828135</td>\n",
              "      <td>0.297556</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.228200</td>\n",
              "      <td>2.576829</td>\n",
              "      <td>0.369288</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.721500</td>\n",
              "      <td>2.509804</td>\n",
              "      <td>0.346440</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.482400</td>\n",
              "      <td>2.340569</td>\n",
              "      <td>0.405420</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.219100</td>\n",
              "      <td>2.322978</td>\n",
              "      <td>0.431987</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.801700</td>\n",
              "      <td>2.297790</td>\n",
              "      <td>0.436238</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.811100</td>\n",
              "      <td>2.339722</td>\n",
              "      <td>0.435175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.557500</td>\n",
              "      <td>2.346094</td>\n",
              "      <td>0.442614</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.446100</td>\n",
              "      <td>2.328964</td>\n",
              "      <td>0.453773</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.318700</td>\n",
              "      <td>2.387123</td>\n",
              "      <td>0.452710</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.225500</td>\n",
              "      <td>2.396320</td>\n",
              "      <td>0.447396</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.212900</td>\n",
              "      <td>2.446562</td>\n",
              "      <td>0.451116</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.167200</td>\n",
              "      <td>2.480164</td>\n",
              "      <td>0.453241</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.140700</td>\n",
              "      <td>2.499099</td>\n",
              "      <td>0.449522</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.138600</td>\n",
              "      <td>2.528399</td>\n",
              "      <td>0.451116</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.105000</td>\n",
              "      <td>2.540300</td>\n",
              "      <td>0.447928</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.093000</td>\n",
              "      <td>2.561310</td>\n",
              "      <td>0.450053</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.099500</td>\n",
              "      <td>2.562459</td>\n",
              "      <td>0.454835</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-500 (score: 0.45483528161530284).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "400 1882 1883\n",
            "# classes in train 38\n",
            "# classes in dev 41\n",
            "# classes in test 40\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:40, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.420400</td>\n",
              "      <td>3.183420</td>\n",
              "      <td>0.196068</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.920400</td>\n",
              "      <td>2.900103</td>\n",
              "      <td>0.251328</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.679400</td>\n",
              "      <td>2.660297</td>\n",
              "      <td>0.314028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.137300</td>\n",
              "      <td>2.519305</td>\n",
              "      <td>0.366631</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.755900</td>\n",
              "      <td>2.443678</td>\n",
              "      <td>0.392136</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.346700</td>\n",
              "      <td>2.402054</td>\n",
              "      <td>0.402232</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.058000</td>\n",
              "      <td>2.353516</td>\n",
              "      <td>0.412859</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.759400</td>\n",
              "      <td>2.361482</td>\n",
              "      <td>0.414453</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.629500</td>\n",
              "      <td>2.416414</td>\n",
              "      <td>0.400106</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.427200</td>\n",
              "      <td>2.414849</td>\n",
              "      <td>0.417109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.370900</td>\n",
              "      <td>2.464211</td>\n",
              "      <td>0.412327</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.323200</td>\n",
              "      <td>2.470637</td>\n",
              "      <td>0.422423</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.218900</td>\n",
              "      <td>2.491259</td>\n",
              "      <td>0.420829</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.175700</td>\n",
              "      <td>2.546290</td>\n",
              "      <td>0.421360</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.156000</td>\n",
              "      <td>2.566342</td>\n",
              "      <td>0.426674</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.150500</td>\n",
              "      <td>2.619205</td>\n",
              "      <td>0.426674</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.102200</td>\n",
              "      <td>2.642443</td>\n",
              "      <td>0.425611</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.076600</td>\n",
              "      <td>2.660656</td>\n",
              "      <td>0.419235</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.112000</td>\n",
              "      <td>2.669976</td>\n",
              "      <td>0.425080</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.109900</td>\n",
              "      <td>2.669099</td>\n",
              "      <td>0.426674</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-375 (score: 0.4266737513283741).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "400 1882 1883\n",
            "# classes in train 33\n",
            "# classes in dev 41\n",
            "# classes in test 41\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:40, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.336500</td>\n",
              "      <td>3.218424</td>\n",
              "      <td>0.195537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.039700</td>\n",
              "      <td>2.923923</td>\n",
              "      <td>0.259299</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.597900</td>\n",
              "      <td>2.688065</td>\n",
              "      <td>0.321467</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.142300</td>\n",
              "      <td>2.497275</td>\n",
              "      <td>0.391073</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.021300</td>\n",
              "      <td>2.408473</td>\n",
              "      <td>0.407545</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.557200</td>\n",
              "      <td>2.386353</td>\n",
              "      <td>0.428799</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.337700</td>\n",
              "      <td>2.332740</td>\n",
              "      <td>0.437301</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.032500</td>\n",
              "      <td>2.334630</td>\n",
              "      <td>0.442083</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.755900</td>\n",
              "      <td>2.257548</td>\n",
              "      <td>0.460680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.650200</td>\n",
              "      <td>2.339005</td>\n",
              "      <td>0.468650</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.461300</td>\n",
              "      <td>2.324004</td>\n",
              "      <td>0.452179</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.401500</td>\n",
              "      <td>2.338417</td>\n",
              "      <td>0.462806</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.331900</td>\n",
              "      <td>2.384988</td>\n",
              "      <td>0.458555</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.197000</td>\n",
              "      <td>2.454885</td>\n",
              "      <td>0.447928</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.173700</td>\n",
              "      <td>2.441020</td>\n",
              "      <td>0.460680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.151000</td>\n",
              "      <td>2.465099</td>\n",
              "      <td>0.461743</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.130700</td>\n",
              "      <td>2.525830</td>\n",
              "      <td>0.447928</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.132600</td>\n",
              "      <td>2.502510</td>\n",
              "      <td>0.459617</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.103400</td>\n",
              "      <td>2.516666</td>\n",
              "      <td>0.456961</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.097500</td>\n",
              "      <td>2.527249</td>\n",
              "      <td>0.457492</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-250 (score: 0.46865037194473963).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "400 1882 1883\n",
            "# classes in train 35\n",
            "# classes in dev 39\n",
            "# classes in test 42\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:40, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.277700</td>\n",
              "      <td>3.231005</td>\n",
              "      <td>0.184910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.823400</td>\n",
              "      <td>2.971889</td>\n",
              "      <td>0.263018</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.578300</td>\n",
              "      <td>2.677901</td>\n",
              "      <td>0.321467</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.815400</td>\n",
              "      <td>2.503471</td>\n",
              "      <td>0.374070</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.564700</td>\n",
              "      <td>2.431044</td>\n",
              "      <td>0.381509</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.294800</td>\n",
              "      <td>2.455089</td>\n",
              "      <td>0.414984</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.940100</td>\n",
              "      <td>2.389544</td>\n",
              "      <td>0.421892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.793400</td>\n",
              "      <td>2.350041</td>\n",
              "      <td>0.427205</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.570900</td>\n",
              "      <td>2.376637</td>\n",
              "      <td>0.445802</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.441300</td>\n",
              "      <td>2.493687</td>\n",
              "      <td>0.428799</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.364500</td>\n",
              "      <td>2.462389</td>\n",
              "      <td>0.415515</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.252900</td>\n",
              "      <td>2.500139</td>\n",
              "      <td>0.435707</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.224500</td>\n",
              "      <td>2.571344</td>\n",
              "      <td>0.436769</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.147500</td>\n",
              "      <td>2.612877</td>\n",
              "      <td>0.439426</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.130500</td>\n",
              "      <td>2.633719</td>\n",
              "      <td>0.437832</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.104600</td>\n",
              "      <td>2.715561</td>\n",
              "      <td>0.425080</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.104200</td>\n",
              "      <td>2.734201</td>\n",
              "      <td>0.438363</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.096200</td>\n",
              "      <td>2.744576</td>\n",
              "      <td>0.441552</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.090500</td>\n",
              "      <td>2.751831</td>\n",
              "      <td>0.442614</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.096200</td>\n",
              "      <td>2.760236</td>\n",
              "      <td>0.439426</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-225 (score: 0.44580233793836344).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "400 1882 1883\n",
            "# classes in train 36\n",
            "# classes in dev 41\n",
            "# classes in test 39\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:43, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.561700</td>\n",
              "      <td>3.206930</td>\n",
              "      <td>0.192880</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.231700</td>\n",
              "      <td>3.092108</td>\n",
              "      <td>0.247078</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.037100</td>\n",
              "      <td>2.848934</td>\n",
              "      <td>0.267269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.577400</td>\n",
              "      <td>2.671996</td>\n",
              "      <td>0.335282</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.015100</td>\n",
              "      <td>2.449120</td>\n",
              "      <td>0.396387</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.439100</td>\n",
              "      <td>2.352765</td>\n",
              "      <td>0.417641</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.272700</td>\n",
              "      <td>2.483296</td>\n",
              "      <td>0.385228</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.880100</td>\n",
              "      <td>2.375380</td>\n",
              "      <td>0.421892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.790400</td>\n",
              "      <td>2.356519</td>\n",
              "      <td>0.433581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.513900</td>\n",
              "      <td>2.436685</td>\n",
              "      <td>0.427205</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.465000</td>\n",
              "      <td>2.437097</td>\n",
              "      <td>0.425611</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.377800</td>\n",
              "      <td>2.497279</td>\n",
              "      <td>0.421360</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.297400</td>\n",
              "      <td>2.528420</td>\n",
              "      <td>0.434113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.213300</td>\n",
              "      <td>2.540598</td>\n",
              "      <td>0.431987</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.246600</td>\n",
              "      <td>2.599110</td>\n",
              "      <td>0.431456</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.185100</td>\n",
              "      <td>2.634126</td>\n",
              "      <td>0.436769</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.123300</td>\n",
              "      <td>2.665287</td>\n",
              "      <td>0.434644</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.148300</td>\n",
              "      <td>2.677024</td>\n",
              "      <td>0.435175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.121300</td>\n",
              "      <td>2.704699</td>\n",
              "      <td>0.429330</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.118000</td>\n",
              "      <td>2.711666</td>\n",
              "      <td>0.433050</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-400 (score: 0.436769394261424).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "200 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:06, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.017200</td>\n",
              "      <td>1.943102</td>\n",
              "      <td>0.242295</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.882600</td>\n",
              "      <td>1.885604</td>\n",
              "      <td>0.242295</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.790000</td>\n",
              "      <td>1.733940</td>\n",
              "      <td>0.322529</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.501400</td>\n",
              "      <td>1.577496</td>\n",
              "      <td>0.448459</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.165000</td>\n",
              "      <td>1.565355</td>\n",
              "      <td>0.448459</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.963900</td>\n",
              "      <td>1.577729</td>\n",
              "      <td>0.484591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.481500</td>\n",
              "      <td>1.640471</td>\n",
              "      <td>0.473433</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.379900</td>\n",
              "      <td>1.811237</td>\n",
              "      <td>0.474495</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.318500</td>\n",
              "      <td>1.798931</td>\n",
              "      <td>0.510627</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.143100</td>\n",
              "      <td>1.968634</td>\n",
              "      <td>0.492561</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.098200</td>\n",
              "      <td>2.070112</td>\n",
              "      <td>0.499469</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.041600</td>\n",
              "      <td>2.168685</td>\n",
              "      <td>0.504251</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.039100</td>\n",
              "      <td>2.304891</td>\n",
              "      <td>0.501063</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.017300</td>\n",
              "      <td>2.378513</td>\n",
              "      <td>0.499469</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.013400</td>\n",
              "      <td>2.425080</td>\n",
              "      <td>0.503719</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.011000</td>\n",
              "      <td>2.466122</td>\n",
              "      <td>0.503719</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.009900</td>\n",
              "      <td>2.486077</td>\n",
              "      <td>0.503188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.008800</td>\n",
              "      <td>2.505417</td>\n",
              "      <td>0.502125</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.008700</td>\n",
              "      <td>2.518810</td>\n",
              "      <td>0.503188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.008200</td>\n",
              "      <td>2.523809</td>\n",
              "      <td>0.502125</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-117 (score: 0.5106269925611052).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "200 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:08, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.059500</td>\n",
              "      <td>1.926887</td>\n",
              "      <td>0.266738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.919300</td>\n",
              "      <td>1.837176</td>\n",
              "      <td>0.358130</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.903100</td>\n",
              "      <td>1.663730</td>\n",
              "      <td>0.413390</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.380500</td>\n",
              "      <td>1.495119</td>\n",
              "      <td>0.477152</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.047200</td>\n",
              "      <td>1.519470</td>\n",
              "      <td>0.490967</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.810100</td>\n",
              "      <td>1.626215</td>\n",
              "      <td>0.485654</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.405300</td>\n",
              "      <td>1.560553</td>\n",
              "      <td>0.522848</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.253800</td>\n",
              "      <td>1.725679</td>\n",
              "      <td>0.502125</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.143200</td>\n",
              "      <td>1.822184</td>\n",
              "      <td>0.517003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.054700</td>\n",
              "      <td>2.102013</td>\n",
              "      <td>0.497875</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.039500</td>\n",
              "      <td>2.148624</td>\n",
              "      <td>0.498937</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.025400</td>\n",
              "      <td>2.153775</td>\n",
              "      <td>0.521254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.015900</td>\n",
              "      <td>2.225486</td>\n",
              "      <td>0.504251</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.010600</td>\n",
              "      <td>2.298549</td>\n",
              "      <td>0.515409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.010000</td>\n",
              "      <td>2.320534</td>\n",
              "      <td>0.515409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.009600</td>\n",
              "      <td>2.360164</td>\n",
              "      <td>0.510627</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.007700</td>\n",
              "      <td>2.404098</td>\n",
              "      <td>0.510096</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.008100</td>\n",
              "      <td>2.414501</td>\n",
              "      <td>0.515409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.007100</td>\n",
              "      <td>2.419100</td>\n",
              "      <td>0.514346</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.007000</td>\n",
              "      <td>2.424375</td>\n",
              "      <td>0.516472</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-91 (score: 0.5228480340063762).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "200 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:02, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.017100</td>\n",
              "      <td>1.894493</td>\n",
              "      <td>0.267269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.910800</td>\n",
              "      <td>1.828404</td>\n",
              "      <td>0.267269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.755200</td>\n",
              "      <td>1.629486</td>\n",
              "      <td>0.416578</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.270400</td>\n",
              "      <td>1.513461</td>\n",
              "      <td>0.464400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.063500</td>\n",
              "      <td>1.576056</td>\n",
              "      <td>0.448990</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.896600</td>\n",
              "      <td>1.498389</td>\n",
              "      <td>0.512752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.441400</td>\n",
              "      <td>1.558421</td>\n",
              "      <td>0.527099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.303400</td>\n",
              "      <td>1.665541</td>\n",
              "      <td>0.505845</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.214600</td>\n",
              "      <td>1.730014</td>\n",
              "      <td>0.525505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.071400</td>\n",
              "      <td>1.781626</td>\n",
              "      <td>0.525505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.048900</td>\n",
              "      <td>1.920781</td>\n",
              "      <td>0.524973</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.024200</td>\n",
              "      <td>2.183494</td>\n",
              "      <td>0.509033</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.026900</td>\n",
              "      <td>2.131054</td>\n",
              "      <td>0.533475</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.011600</td>\n",
              "      <td>2.198333</td>\n",
              "      <td>0.535069</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.011400</td>\n",
              "      <td>2.255089</td>\n",
              "      <td>0.531881</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.009700</td>\n",
              "      <td>2.275043</td>\n",
              "      <td>0.534538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.008200</td>\n",
              "      <td>2.287066</td>\n",
              "      <td>0.534538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.008300</td>\n",
              "      <td>2.302487</td>\n",
              "      <td>0.533475</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.007700</td>\n",
              "      <td>2.317101</td>\n",
              "      <td>0.536663</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.007300</td>\n",
              "      <td>2.320747</td>\n",
              "      <td>0.534538</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-247 (score: 0.536663124335813).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "200 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:02, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.995900</td>\n",
              "      <td>1.926051</td>\n",
              "      <td>0.255579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.855900</td>\n",
              "      <td>1.864178</td>\n",
              "      <td>0.320404</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.906000</td>\n",
              "      <td>1.761038</td>\n",
              "      <td>0.300744</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.530500</td>\n",
              "      <td>1.564557</td>\n",
              "      <td>0.459617</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.188800</td>\n",
              "      <td>1.383003</td>\n",
              "      <td>0.528693</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.959500</td>\n",
              "      <td>1.444349</td>\n",
              "      <td>0.528162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.370200</td>\n",
              "      <td>1.553294</td>\n",
              "      <td>0.527099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.255200</td>\n",
              "      <td>1.703556</td>\n",
              "      <td>0.518066</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.191500</td>\n",
              "      <td>1.727714</td>\n",
              "      <td>0.534006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.092700</td>\n",
              "      <td>1.895508</td>\n",
              "      <td>0.522317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.043300</td>\n",
              "      <td>2.098443</td>\n",
              "      <td>0.512752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.085200</td>\n",
              "      <td>2.120096</td>\n",
              "      <td>0.528162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.022400</td>\n",
              "      <td>2.101261</td>\n",
              "      <td>0.527099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.016300</td>\n",
              "      <td>2.227777</td>\n",
              "      <td>0.527630</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.012700</td>\n",
              "      <td>2.255225</td>\n",
              "      <td>0.522317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.010400</td>\n",
              "      <td>2.275755</td>\n",
              "      <td>0.525505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.010200</td>\n",
              "      <td>2.303538</td>\n",
              "      <td>0.521785</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.009100</td>\n",
              "      <td>2.318013</td>\n",
              "      <td>0.521254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.009200</td>\n",
              "      <td>2.326589</td>\n",
              "      <td>0.521785</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.009100</td>\n",
              "      <td>2.331437</td>\n",
              "      <td>0.520723</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-117 (score: 0.5340063761955367).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "200 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 200\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 260\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='260' max='260' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [260/260 07:08, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.053200</td>\n",
              "      <td>1.956157</td>\n",
              "      <td>0.164187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.947500</td>\n",
              "      <td>1.804451</td>\n",
              "      <td>0.382040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.828100</td>\n",
              "      <td>1.639905</td>\n",
              "      <td>0.403294</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.258000</td>\n",
              "      <td>1.583122</td>\n",
              "      <td>0.422954</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.094500</td>\n",
              "      <td>1.510847</td>\n",
              "      <td>0.475027</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.863900</td>\n",
              "      <td>1.478758</td>\n",
              "      <td>0.522317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.402700</td>\n",
              "      <td>1.537051</td>\n",
              "      <td>0.530287</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.277500</td>\n",
              "      <td>1.709229</td>\n",
              "      <td>0.515940</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.164400</td>\n",
              "      <td>1.780277</td>\n",
              "      <td>0.512752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.085000</td>\n",
              "      <td>1.943420</td>\n",
              "      <td>0.516472</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.034000</td>\n",
              "      <td>1.998989</td>\n",
              "      <td>0.530818</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.054700</td>\n",
              "      <td>2.204335</td>\n",
              "      <td>0.531350</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.044500</td>\n",
              "      <td>2.244324</td>\n",
              "      <td>0.535600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.011800</td>\n",
              "      <td>2.283870</td>\n",
              "      <td>0.529224</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.024800</td>\n",
              "      <td>2.383059</td>\n",
              "      <td>0.521785</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.009500</td>\n",
              "      <td>2.370011</td>\n",
              "      <td>0.532944</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.008500</td>\n",
              "      <td>2.369165</td>\n",
              "      <td>0.539851</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.007700</td>\n",
              "      <td>2.371448</td>\n",
              "      <td>0.544102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.007700</td>\n",
              "      <td>2.378648</td>\n",
              "      <td>0.543571</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.006900</td>\n",
              "      <td>2.380937</td>\n",
              "      <td>0.543039</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-13\n",
            "Configuration saved in ./results/checkpoint-13/config.json\n",
            "Model weights saved in ./results/checkpoint-13/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-26\n",
            "Configuration saved in ./results/checkpoint-26/config.json\n",
            "Model weights saved in ./results/checkpoint-26/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-39\n",
            "Configuration saved in ./results/checkpoint-39/config.json\n",
            "Model weights saved in ./results/checkpoint-39/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-13] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-52\n",
            "Configuration saved in ./results/checkpoint-52/config.json\n",
            "Model weights saved in ./results/checkpoint-52/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-26] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-65\n",
            "Configuration saved in ./results/checkpoint-65/config.json\n",
            "Model weights saved in ./results/checkpoint-65/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-39] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-78\n",
            "Configuration saved in ./results/checkpoint-78/config.json\n",
            "Model weights saved in ./results/checkpoint-78/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-52] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-91\n",
            "Configuration saved in ./results/checkpoint-91/config.json\n",
            "Model weights saved in ./results/checkpoint-91/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-65] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-104\n",
            "Configuration saved in ./results/checkpoint-104/config.json\n",
            "Model weights saved in ./results/checkpoint-104/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-78] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-117\n",
            "Configuration saved in ./results/checkpoint-117/config.json\n",
            "Model weights saved in ./results/checkpoint-117/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-104] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-130\n",
            "Configuration saved in ./results/checkpoint-130/config.json\n",
            "Model weights saved in ./results/checkpoint-130/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-117] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-143\n",
            "Configuration saved in ./results/checkpoint-143/config.json\n",
            "Model weights saved in ./results/checkpoint-143/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-91] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-156\n",
            "Configuration saved in ./results/checkpoint-156/config.json\n",
            "Model weights saved in ./results/checkpoint-156/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-130] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-169\n",
            "Configuration saved in ./results/checkpoint-169/config.json\n",
            "Model weights saved in ./results/checkpoint-169/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-143] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-182\n",
            "Configuration saved in ./results/checkpoint-182/config.json\n",
            "Model weights saved in ./results/checkpoint-182/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-156] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-195\n",
            "Configuration saved in ./results/checkpoint-195/config.json\n",
            "Model weights saved in ./results/checkpoint-195/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-182] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-208\n",
            "Configuration saved in ./results/checkpoint-208/config.json\n",
            "Model weights saved in ./results/checkpoint-208/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-195] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-221\n",
            "Configuration saved in ./results/checkpoint-221/config.json\n",
            "Model weights saved in ./results/checkpoint-221/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-169] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-234\n",
            "Configuration saved in ./results/checkpoint-234/config.json\n",
            "Model weights saved in ./results/checkpoint-234/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-208] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-221] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-260\n",
            "Configuration saved in ./results/checkpoint-260/config.json\n",
            "Model weights saved in ./results/checkpoint-260/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-234 (score: 0.5441020191285866).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "300 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:53, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.017300</td>\n",
              "      <td>1.915356</td>\n",
              "      <td>0.194474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.759000</td>\n",
              "      <td>1.718211</td>\n",
              "      <td>0.311902</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.505900</td>\n",
              "      <td>1.527857</td>\n",
              "      <td>0.444740</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.041000</td>\n",
              "      <td>1.442086</td>\n",
              "      <td>0.487779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.703100</td>\n",
              "      <td>1.486028</td>\n",
              "      <td>0.498937</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.388900</td>\n",
              "      <td>1.660009</td>\n",
              "      <td>0.517003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.190800</td>\n",
              "      <td>1.871873</td>\n",
              "      <td>0.505313</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.121300</td>\n",
              "      <td>2.028041</td>\n",
              "      <td>0.497343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.039200</td>\n",
              "      <td>2.164007</td>\n",
              "      <td>0.510627</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.014600</td>\n",
              "      <td>2.360769</td>\n",
              "      <td>0.513815</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.012100</td>\n",
              "      <td>2.486615</td>\n",
              "      <td>0.512752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.008500</td>\n",
              "      <td>2.536839</td>\n",
              "      <td>0.522317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.006600</td>\n",
              "      <td>2.598193</td>\n",
              "      <td>0.522848</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.005600</td>\n",
              "      <td>2.636643</td>\n",
              "      <td>0.520723</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.005400</td>\n",
              "      <td>2.664160</td>\n",
              "      <td>0.524973</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.004600</td>\n",
              "      <td>2.696240</td>\n",
              "      <td>0.520723</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.717276</td>\n",
              "      <td>0.520191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.730316</td>\n",
              "      <td>0.520191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.003900</td>\n",
              "      <td>2.737976</td>\n",
              "      <td>0.521254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.004000</td>\n",
              "      <td>2.740930</td>\n",
              "      <td>0.520191</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-234] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-260] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-285 (score: 0.5249734325185972).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "300 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:54, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.035400</td>\n",
              "      <td>1.853607</td>\n",
              "      <td>0.267800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.832300</td>\n",
              "      <td>1.622635</td>\n",
              "      <td>0.438895</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.419900</td>\n",
              "      <td>1.448552</td>\n",
              "      <td>0.468119</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.996200</td>\n",
              "      <td>1.439003</td>\n",
              "      <td>0.537194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.638600</td>\n",
              "      <td>1.468036</td>\n",
              "      <td>0.527630</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.411700</td>\n",
              "      <td>1.624780</td>\n",
              "      <td>0.534006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.252600</td>\n",
              "      <td>1.820151</td>\n",
              "      <td>0.499469</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.095300</td>\n",
              "      <td>2.046814</td>\n",
              "      <td>0.495749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.080700</td>\n",
              "      <td>2.029547</td>\n",
              "      <td>0.517535</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.021000</td>\n",
              "      <td>2.318681</td>\n",
              "      <td>0.520191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.021300</td>\n",
              "      <td>2.435732</td>\n",
              "      <td>0.518597</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.047200</td>\n",
              "      <td>2.581680</td>\n",
              "      <td>0.511690</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.008100</td>\n",
              "      <td>2.724349</td>\n",
              "      <td>0.498406</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.006700</td>\n",
              "      <td>2.625994</td>\n",
              "      <td>0.525505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.006700</td>\n",
              "      <td>2.682714</td>\n",
              "      <td>0.522317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.005500</td>\n",
              "      <td>2.718161</td>\n",
              "      <td>0.518597</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.005100</td>\n",
              "      <td>2.726978</td>\n",
              "      <td>0.521254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.004800</td>\n",
              "      <td>2.735348</td>\n",
              "      <td>0.521785</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.004400</td>\n",
              "      <td>2.745908</td>\n",
              "      <td>0.520723</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.004200</td>\n",
              "      <td>2.750406</td>\n",
              "      <td>0.521254</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-76 (score: 0.5371944739638682).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "300 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:51, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.053100</td>\n",
              "      <td>1.880952</td>\n",
              "      <td>0.287991</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.788400</td>\n",
              "      <td>1.703997</td>\n",
              "      <td>0.376196</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.452700</td>\n",
              "      <td>1.512457</td>\n",
              "      <td>0.486716</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.944500</td>\n",
              "      <td>1.562150</td>\n",
              "      <td>0.483528</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.597500</td>\n",
              "      <td>1.541220</td>\n",
              "      <td>0.506376</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.276400</td>\n",
              "      <td>1.578075</td>\n",
              "      <td>0.528162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.132100</td>\n",
              "      <td>1.807051</td>\n",
              "      <td>0.518066</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.056200</td>\n",
              "      <td>2.103063</td>\n",
              "      <td>0.515409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.056400</td>\n",
              "      <td>2.264840</td>\n",
              "      <td>0.526036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.010500</td>\n",
              "      <td>2.567545</td>\n",
              "      <td>0.498937</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.009200</td>\n",
              "      <td>2.533570</td>\n",
              "      <td>0.516472</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.006500</td>\n",
              "      <td>2.534084</td>\n",
              "      <td>0.526036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.005300</td>\n",
              "      <td>2.603456</td>\n",
              "      <td>0.526567</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.004700</td>\n",
              "      <td>2.654147</td>\n",
              "      <td>0.523911</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.674987</td>\n",
              "      <td>0.526036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.004200</td>\n",
              "      <td>2.699073</td>\n",
              "      <td>0.526036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.003900</td>\n",
              "      <td>2.720587</td>\n",
              "      <td>0.526036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.003800</td>\n",
              "      <td>2.733615</td>\n",
              "      <td>0.526036</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.003600</td>\n",
              "      <td>2.740501</td>\n",
              "      <td>0.526567</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.003600</td>\n",
              "      <td>2.741858</td>\n",
              "      <td>0.526567</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-114 (score: 0.5281615302869288).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "300 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:52, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.044700</td>\n",
              "      <td>1.925716</td>\n",
              "      <td>0.255579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.853400</td>\n",
              "      <td>1.757751</td>\n",
              "      <td>0.393199</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.516700</td>\n",
              "      <td>1.535486</td>\n",
              "      <td>0.448990</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.078500</td>\n",
              "      <td>1.435240</td>\n",
              "      <td>0.522317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.669200</td>\n",
              "      <td>1.480019</td>\n",
              "      <td>0.515940</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.320600</td>\n",
              "      <td>1.622540</td>\n",
              "      <td>0.534538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.169500</td>\n",
              "      <td>1.808301</td>\n",
              "      <td>0.512752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.084100</td>\n",
              "      <td>1.891993</td>\n",
              "      <td>0.534538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.054300</td>\n",
              "      <td>2.121623</td>\n",
              "      <td>0.532412</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.015700</td>\n",
              "      <td>2.242105</td>\n",
              "      <td>0.531350</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.019600</td>\n",
              "      <td>2.439630</td>\n",
              "      <td>0.532944</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.008600</td>\n",
              "      <td>2.431101</td>\n",
              "      <td>0.537726</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.007500</td>\n",
              "      <td>2.484043</td>\n",
              "      <td>0.539320</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.005900</td>\n",
              "      <td>2.556237</td>\n",
              "      <td>0.538789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.005700</td>\n",
              "      <td>2.581295</td>\n",
              "      <td>0.537194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.004900</td>\n",
              "      <td>2.599207</td>\n",
              "      <td>0.539320</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.004800</td>\n",
              "      <td>2.615087</td>\n",
              "      <td>0.539851</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.004700</td>\n",
              "      <td>2.629185</td>\n",
              "      <td>0.540383</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.004400</td>\n",
              "      <td>2.636232</td>\n",
              "      <td>0.543039</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.004300</td>\n",
              "      <td>2.640498</td>\n",
              "      <td>0.543039</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-361 (score: 0.5430393198724761).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "300 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 300\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 380\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='380' max='380' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [380/380 07:54, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.059700</td>\n",
              "      <td>1.883418</td>\n",
              "      <td>0.236982</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.859100</td>\n",
              "      <td>1.638963</td>\n",
              "      <td>0.390542</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.381700</td>\n",
              "      <td>1.539201</td>\n",
              "      <td>0.461211</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.963600</td>\n",
              "      <td>1.459565</td>\n",
              "      <td>0.534006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.520900</td>\n",
              "      <td>1.574273</td>\n",
              "      <td>0.528162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.412800</td>\n",
              "      <td>1.620064</td>\n",
              "      <td>0.534538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.196400</td>\n",
              "      <td>1.737319</td>\n",
              "      <td>0.549947</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.085900</td>\n",
              "      <td>2.125861</td>\n",
              "      <td>0.513284</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.043100</td>\n",
              "      <td>2.287673</td>\n",
              "      <td>0.517535</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.016300</td>\n",
              "      <td>2.314350</td>\n",
              "      <td>0.538789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.023200</td>\n",
              "      <td>2.455081</td>\n",
              "      <td>0.542508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.009800</td>\n",
              "      <td>2.642855</td>\n",
              "      <td>0.527630</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.006700</td>\n",
              "      <td>2.606210</td>\n",
              "      <td>0.532944</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.005400</td>\n",
              "      <td>2.651434</td>\n",
              "      <td>0.537194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.004900</td>\n",
              "      <td>2.693531</td>\n",
              "      <td>0.532412</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.723267</td>\n",
              "      <td>0.531350</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.004000</td>\n",
              "      <td>2.736109</td>\n",
              "      <td>0.535600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.004200</td>\n",
              "      <td>2.737974</td>\n",
              "      <td>0.533475</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.003700</td>\n",
              "      <td>2.743168</td>\n",
              "      <td>0.534538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.003800</td>\n",
              "      <td>2.746512</td>\n",
              "      <td>0.534006</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-19\n",
            "Configuration saved in ./results/checkpoint-19/config.json\n",
            "Model weights saved in ./results/checkpoint-19/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-38\n",
            "Configuration saved in ./results/checkpoint-38/config.json\n",
            "Model weights saved in ./results/checkpoint-38/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-57\n",
            "Configuration saved in ./results/checkpoint-57/config.json\n",
            "Model weights saved in ./results/checkpoint-57/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-19] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-76\n",
            "Configuration saved in ./results/checkpoint-76/config.json\n",
            "Model weights saved in ./results/checkpoint-76/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-38] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-95\n",
            "Configuration saved in ./results/checkpoint-95/config.json\n",
            "Model weights saved in ./results/checkpoint-95/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-57] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-114\n",
            "Configuration saved in ./results/checkpoint-114/config.json\n",
            "Model weights saved in ./results/checkpoint-114/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-76] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-133\n",
            "Configuration saved in ./results/checkpoint-133/config.json\n",
            "Model weights saved in ./results/checkpoint-133/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-95] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-152\n",
            "Configuration saved in ./results/checkpoint-152/config.json\n",
            "Model weights saved in ./results/checkpoint-152/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-114] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-171\n",
            "Configuration saved in ./results/checkpoint-171/config.json\n",
            "Model weights saved in ./results/checkpoint-171/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-152] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-190\n",
            "Configuration saved in ./results/checkpoint-190/config.json\n",
            "Model weights saved in ./results/checkpoint-190/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-171] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-209\n",
            "Configuration saved in ./results/checkpoint-209/config.json\n",
            "Model weights saved in ./results/checkpoint-209/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-190] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-228\n",
            "Configuration saved in ./results/checkpoint-228/config.json\n",
            "Model weights saved in ./results/checkpoint-228/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-209] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-247\n",
            "Configuration saved in ./results/checkpoint-247/config.json\n",
            "Model weights saved in ./results/checkpoint-247/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-228] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-266\n",
            "Configuration saved in ./results/checkpoint-266/config.json\n",
            "Model weights saved in ./results/checkpoint-266/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-247] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-285\n",
            "Configuration saved in ./results/checkpoint-285/config.json\n",
            "Model weights saved in ./results/checkpoint-285/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-266] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-304\n",
            "Configuration saved in ./results/checkpoint-304/config.json\n",
            "Model weights saved in ./results/checkpoint-304/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-285] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-323\n",
            "Configuration saved in ./results/checkpoint-323/config.json\n",
            "Model weights saved in ./results/checkpoint-323/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-304] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-342\n",
            "Configuration saved in ./results/checkpoint-342/config.json\n",
            "Model weights saved in ./results/checkpoint-342/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-323] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-361\n",
            "Configuration saved in ./results/checkpoint-361/config.json\n",
            "Model weights saved in ./results/checkpoint-361/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-342] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-380\n",
            "Configuration saved in ./results/checkpoint-380/config.json\n",
            "Model weights saved in ./results/checkpoint-380/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-361] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-133 (score: 0.5499468650371945).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "400 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:39, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.913500</td>\n",
              "      <td>1.872633</td>\n",
              "      <td>0.246546</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.688500</td>\n",
              "      <td>1.619704</td>\n",
              "      <td>0.414984</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.345300</td>\n",
              "      <td>1.475760</td>\n",
              "      <td>0.476089</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.959600</td>\n",
              "      <td>1.398114</td>\n",
              "      <td>0.534006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.725900</td>\n",
              "      <td>1.462917</td>\n",
              "      <td>0.543039</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.296900</td>\n",
              "      <td>1.567623</td>\n",
              "      <td>0.559511</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.237600</td>\n",
              "      <td>1.671318</td>\n",
              "      <td>0.551541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.092100</td>\n",
              "      <td>1.956201</td>\n",
              "      <td>0.538789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.079400</td>\n",
              "      <td>2.213268</td>\n",
              "      <td>0.529756</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.037100</td>\n",
              "      <td>2.247084</td>\n",
              "      <td>0.548353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.025700</td>\n",
              "      <td>2.401659</td>\n",
              "      <td>0.552072</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.006700</td>\n",
              "      <td>2.500400</td>\n",
              "      <td>0.547290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.012500</td>\n",
              "      <td>2.622443</td>\n",
              "      <td>0.541977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.004200</td>\n",
              "      <td>2.598059</td>\n",
              "      <td>0.546759</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.004200</td>\n",
              "      <td>2.624339</td>\n",
              "      <td>0.552072</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.003600</td>\n",
              "      <td>2.645604</td>\n",
              "      <td>0.550478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.003600</td>\n",
              "      <td>2.670373</td>\n",
              "      <td>0.549947</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>2.686526</td>\n",
              "      <td>0.551010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>2.696722</td>\n",
              "      <td>0.551541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.003100</td>\n",
              "      <td>2.699128</td>\n",
              "      <td>0.551010</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-133] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-380] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-150 (score: 0.5595111583421891).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "400 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='9' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [  9/500 00:02 < 03:08, 2.60 it/s, Epoch 0.32/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:40, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.009000</td>\n",
              "      <td>1.826857</td>\n",
              "      <td>0.361318</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.670100</td>\n",
              "      <td>1.511216</td>\n",
              "      <td>0.466525</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.234100</td>\n",
              "      <td>1.488632</td>\n",
              "      <td>0.472370</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.936300</td>\n",
              "      <td>1.393727</td>\n",
              "      <td>0.542508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.623600</td>\n",
              "      <td>1.464176</td>\n",
              "      <td>0.535600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.491800</td>\n",
              "      <td>1.601031</td>\n",
              "      <td>0.522317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.239600</td>\n",
              "      <td>1.896017</td>\n",
              "      <td>0.505845</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.140500</td>\n",
              "      <td>1.933705</td>\n",
              "      <td>0.535069</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.098300</td>\n",
              "      <td>2.063449</td>\n",
              "      <td>0.541977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.052500</td>\n",
              "      <td>2.348956</td>\n",
              "      <td>0.501063</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.071600</td>\n",
              "      <td>2.382513</td>\n",
              "      <td>0.546227</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.020700</td>\n",
              "      <td>2.471345</td>\n",
              "      <td>0.509033</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.047200</td>\n",
              "      <td>2.492152</td>\n",
              "      <td>0.544102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.015500</td>\n",
              "      <td>2.579589</td>\n",
              "      <td>0.525505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.005500</td>\n",
              "      <td>2.562588</td>\n",
              "      <td>0.551541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.010200</td>\n",
              "      <td>2.615805</td>\n",
              "      <td>0.546227</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.005700</td>\n",
              "      <td>2.675401</td>\n",
              "      <td>0.534006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.682630</td>\n",
              "      <td>0.536663</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.004100</td>\n",
              "      <td>2.689532</td>\n",
              "      <td>0.539320</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.692753</td>\n",
              "      <td>0.538789</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-375 (score: 0.5515409139213603).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# classes 8\n",
            "400 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:39, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.901400</td>\n",
              "      <td>1.867552</td>\n",
              "      <td>0.345909</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.662700</td>\n",
              "      <td>1.586032</td>\n",
              "      <td>0.420829</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.350000</td>\n",
              "      <td>1.525296</td>\n",
              "      <td>0.475558</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.121800</td>\n",
              "      <td>1.501122</td>\n",
              "      <td>0.531881</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.608500</td>\n",
              "      <td>1.771485</td>\n",
              "      <td>0.458555</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.415900</td>\n",
              "      <td>1.555267</td>\n",
              "      <td>0.541445</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.266500</td>\n",
              "      <td>1.694039</td>\n",
              "      <td>0.539320</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.116000</td>\n",
              "      <td>1.934513</td>\n",
              "      <td>0.540914</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.041200</td>\n",
              "      <td>2.190274</td>\n",
              "      <td>0.527630</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.035100</td>\n",
              "      <td>2.277709</td>\n",
              "      <td>0.538257</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.015800</td>\n",
              "      <td>2.352413</td>\n",
              "      <td>0.556323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.018600</td>\n",
              "      <td>2.428463</td>\n",
              "      <td>0.548884</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.005600</td>\n",
              "      <td>2.455173</td>\n",
              "      <td>0.553135</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.524096</td>\n",
              "      <td>0.557386</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.004300</td>\n",
              "      <td>2.535587</td>\n",
              "      <td>0.556323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.003800</td>\n",
              "      <td>2.560637</td>\n",
              "      <td>0.555260</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.003600</td>\n",
              "      <td>2.583357</td>\n",
              "      <td>0.557917</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.003600</td>\n",
              "      <td>2.603131</td>\n",
              "      <td>0.556854</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>2.614229</td>\n",
              "      <td>0.558448</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>2.615336</td>\n",
              "      <td>0.558448</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-475 (score: 0.5584484590860787).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# classes 8\n",
            "400 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:39, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.924200</td>\n",
              "      <td>1.854954</td>\n",
              "      <td>0.264612</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.591400</td>\n",
              "      <td>1.517027</td>\n",
              "      <td>0.472370</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.188500</td>\n",
              "      <td>1.435185</td>\n",
              "      <td>0.498406</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.808100</td>\n",
              "      <td>1.493960</td>\n",
              "      <td>0.503188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.574100</td>\n",
              "      <td>1.474471</td>\n",
              "      <td>0.548353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.280200</td>\n",
              "      <td>1.729007</td>\n",
              "      <td>0.535069</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.225900</td>\n",
              "      <td>1.770037</td>\n",
              "      <td>0.539851</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.071600</td>\n",
              "      <td>2.054895</td>\n",
              "      <td>0.558980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.050400</td>\n",
              "      <td>2.278000</td>\n",
              "      <td>0.546759</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.019300</td>\n",
              "      <td>2.403611</td>\n",
              "      <td>0.548884</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.008200</td>\n",
              "      <td>2.441226</td>\n",
              "      <td>0.554198</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.005800</td>\n",
              "      <td>2.537520</td>\n",
              "      <td>0.551541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.004800</td>\n",
              "      <td>2.645496</td>\n",
              "      <td>0.548884</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.004100</td>\n",
              "      <td>2.612873</td>\n",
              "      <td>0.558980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.003500</td>\n",
              "      <td>2.651583</td>\n",
              "      <td>0.553135</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>2.739360</td>\n",
              "      <td>0.548884</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>2.741282</td>\n",
              "      <td>0.552604</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.003000</td>\n",
              "      <td>2.744802</td>\n",
              "      <td>0.554198</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.002900</td>\n",
              "      <td>2.749070</td>\n",
              "      <td>0.553666</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.002900</td>\n",
              "      <td>2.749798</td>\n",
              "      <td>0.553135</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-225] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-200 (score: 0.5589798087141339).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# classes 8\n",
            "400 1882 1883\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 400\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 500\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [500/500 08:40, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.987500</td>\n",
              "      <td>1.928288</td>\n",
              "      <td>0.200850</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.612800</td>\n",
              "      <td>1.566792</td>\n",
              "      <td>0.448990</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.324800</td>\n",
              "      <td>1.462904</td>\n",
              "      <td>0.498937</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.941100</td>\n",
              "      <td>1.459053</td>\n",
              "      <td>0.539851</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.604500</td>\n",
              "      <td>1.508209</td>\n",
              "      <td>0.538789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.442300</td>\n",
              "      <td>1.749491</td>\n",
              "      <td>0.520191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.345300</td>\n",
              "      <td>2.000509</td>\n",
              "      <td>0.502657</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.157900</td>\n",
              "      <td>2.102963</td>\n",
              "      <td>0.525505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.037100</td>\n",
              "      <td>2.146791</td>\n",
              "      <td>0.554729</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.128200</td>\n",
              "      <td>2.391114</td>\n",
              "      <td>0.535069</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.010400</td>\n",
              "      <td>2.613556</td>\n",
              "      <td>0.538789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.044200</td>\n",
              "      <td>2.641001</td>\n",
              "      <td>0.543571</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.006000</td>\n",
              "      <td>2.705116</td>\n",
              "      <td>0.549947</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.004300</td>\n",
              "      <td>2.736521</td>\n",
              "      <td>0.549947</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.004200</td>\n",
              "      <td>2.781639</td>\n",
              "      <td>0.544633</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.003600</td>\n",
              "      <td>2.820933</td>\n",
              "      <td>0.549947</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>2.844774</td>\n",
              "      <td>0.548353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.003200</td>\n",
              "      <td>2.859853</td>\n",
              "      <td>0.550478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.003200</td>\n",
              "      <td>2.865365</td>\n",
              "      <td>0.548884</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.003200</td>\n",
              "      <td>2.867691</td>\n",
              "      <td>0.549416</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-25\n",
            "Configuration saved in ./results/checkpoint-25/config.json\n",
            "Model weights saved in ./results/checkpoint-25/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-50\n",
            "Configuration saved in ./results/checkpoint-50/config.json\n",
            "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-500] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-75\n",
            "Configuration saved in ./results/checkpoint-75/config.json\n",
            "Model weights saved in ./results/checkpoint-75/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-25] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-100\n",
            "Configuration saved in ./results/checkpoint-100/config.json\n",
            "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-50] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-125\n",
            "Configuration saved in ./results/checkpoint-125/config.json\n",
            "Model weights saved in ./results/checkpoint-125/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-75] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-150\n",
            "Configuration saved in ./results/checkpoint-150/config.json\n",
            "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-125] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-175\n",
            "Configuration saved in ./results/checkpoint-175/config.json\n",
            "Model weights saved in ./results/checkpoint-175/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-150] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-200\n",
            "Configuration saved in ./results/checkpoint-200/config.json\n",
            "Model weights saved in ./results/checkpoint-200/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-175] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-225\n",
            "Configuration saved in ./results/checkpoint-225/config.json\n",
            "Model weights saved in ./results/checkpoint-225/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-100] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-250\n",
            "Configuration saved in ./results/checkpoint-250/config.json\n",
            "Model weights saved in ./results/checkpoint-250/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-200] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-275\n",
            "Configuration saved in ./results/checkpoint-275/config.json\n",
            "Model weights saved in ./results/checkpoint-275/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-250] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-300\n",
            "Configuration saved in ./results/checkpoint-300/config.json\n",
            "Model weights saved in ./results/checkpoint-300/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-275] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-325\n",
            "Configuration saved in ./results/checkpoint-325/config.json\n",
            "Model weights saved in ./results/checkpoint-325/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-300] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-350\n",
            "Configuration saved in ./results/checkpoint-350/config.json\n",
            "Model weights saved in ./results/checkpoint-350/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-325] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-375\n",
            "Configuration saved in ./results/checkpoint-375/config.json\n",
            "Model weights saved in ./results/checkpoint-375/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-350] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-400\n",
            "Configuration saved in ./results/checkpoint-400/config.json\n",
            "Model weights saved in ./results/checkpoint-400/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-375] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-425\n",
            "Configuration saved in ./results/checkpoint-425/config.json\n",
            "Model weights saved in ./results/checkpoint-425/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-400] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-450\n",
            "Configuration saved in ./results/checkpoint-450/config.json\n",
            "Model weights saved in ./results/checkpoint-450/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-425] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-475\n",
            "Configuration saved in ./results/checkpoint-475/config.json\n",
            "Model weights saved in ./results/checkpoint-475/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-450] due to args.save_total_limit\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1882\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Deleting older checkpoint [results/checkpoint-475] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-225 (score: 0.5547290116896918).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 1883\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        }
      ],
      "source": [
        "from collections import defaultdict\n",
        "import csv\n",
        "import random\n",
        "import time\n",
        "\n",
        "from datasets import load_metric\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "import torch\n",
        "from transformers import RobertaTokenizerFast\n",
        "\n",
        "\n",
        "def compute_metrics(eval_preds):\n",
        "    metric = load_metric(\"accuracy\")\n",
        "    logits, labels = eval_preds\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    return metric.compute(predictions=predictions, references=labels)\n",
        "\n",
        "  \n",
        "directory = \"./data_and_models/\"\n",
        "mlength = 512\n",
        "start = time.time()\n",
        "tokenizer = RobertaTokenizerFast.from_pretrained('roberta-base')\n",
        "\n",
        "class PSCDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "def top_k_accuracy(top_k, predictions, labels):\n",
        "  assert len(predictions) == len(labels)\n",
        "  total = 0\n",
        "  correct = 0\n",
        "  for i in range(len(predictions)):\n",
        "    total += 1\n",
        "    prediction = []\n",
        "    for j, k in enumerate(predictions[i]):\n",
        "      prediction.append([j, k])\n",
        "    prediction.sort(key = lambda x: -x[1])\n",
        "    for j, _ in prediction[:top_k]:\n",
        "      if j == labels[i]:\n",
        "        correct += 1\n",
        "        break\n",
        "  return correct/total\n",
        "\n",
        "tasks = {\n",
        "    \"44\": {\n",
        "        \"number_of_labels\": 42,\n",
        "         \"label_column\": 1,\n",
        "    },\n",
        "    \"8\": {\n",
        "        \"number_of_labels\": 8,\n",
        "        \"label_column\": 2,\n",
        "    }\n",
        "}\n",
        "\n",
        "def compute_task(task):\n",
        "  results = {}\n",
        "  for train_size in [200, 300, 400]:\n",
        "    t1 =[]\n",
        "    seeds = range(11, 16)\n",
        "    nclasses = tasks[task][\"number_of_labels\"]\n",
        "    tasks[task][\"label_column\"]\n",
        "    epochs = 20\n",
        "    learning_rate = 4e-5\n",
        "    for seed in seeds:\n",
        "      np.random.seed(seed)\n",
        "      torch.manual_seed(seed)\n",
        "      random.seed(seed)\n",
        "\n",
        "      index = -1\n",
        "      classes = {}\n",
        "      texts = []\n",
        "      labels = []\n",
        "      lm_reverse_mapper = {}\n",
        "      with open(directory + \"target_corpus.csv\") as doc:\n",
        "        reader = csv.reader(doc)\n",
        "        next(reader)\n",
        "        for row in reader:\n",
        "          topic = row[tasks[task][\"label_column\"]]\n",
        "          if topic not in classes:\n",
        "            index += 1\n",
        "            classes[topic] = index\n",
        "            lm_reverse_mapper[index] = topic.capitalize()\n",
        "          labels.append(classes[topic])\n",
        "          texts.append(row[0])\n",
        "      print(\"# classes\", len(classes))\n",
        "      test_size = (4165 - 400)//2 + 1\n",
        "      dev_size = (4165 - 400)//2\n",
        "      X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=test_size, random_state=seed)\n",
        "      X_train, X_dev, y_train, y_dev = train_test_split(X_train, y_train, test_size=dev_size, random_state=seed)\n",
        "      if train_size < 400: \n",
        "        _, X_train, _, y_train = train_test_split(X_train, y_train, test_size = train_size, random_state = seed)\n",
        "      print(len(X_train), len(X_dev), len(X_test))\n",
        "      print(\"# classes in train\", len(set(y_train)))\n",
        "      print(\"# classes in dev\", len(set(y_dev)))\n",
        "      print(\"# classes in test\", len(set(y_test)))\n",
        "\n",
        "\n",
        "      train_encodings = tokenizer(X_train, truncation=True, padding=True, max_length=mlength)\n",
        "      dev_encodings = tokenizer(X_dev, truncation=True, padding=True, max_length = mlength)\n",
        "      test_encodings = tokenizer(X_test, truncation=True, padding=True, max_length= mlength)\n",
        "\n",
        "\n",
        "      train_dataset = PSCDataset(train_encodings, y_train)\n",
        "      dev_dataset = PSCDataset(dev_encodings, y_dev)\n",
        "      test_dataset = PSCDataset(test_encodings, y_test)\n",
        "\n",
        "      from transformers import RobertaForSequenceClassification, TrainingArguments, Trainer\n",
        "\n",
        "      training_args = TrainingArguments(\n",
        "          output_dir=\"./results\",          # output directory\n",
        "          num_train_epochs=epochs,         # total number of training epochs\n",
        "          per_device_train_batch_size=16,  # batch size per device during training\n",
        "          per_device_eval_batch_size=64,   # batch size for evaluation\n",
        "          warmup_steps=0,                  # number of warmup steps for learning rate scheduler\n",
        "          weight_decay=0.01,               # strength of weight decay\n",
        "          logging_dir='./logs',            # directory for storing logs\n",
        "          logging_steps=10,\n",
        "          learning_rate = learning_rate,\n",
        "          save_strategy= \"epoch\",\n",
        "          evaluation_strategy=\"epoch\",\n",
        "          load_best_model_at_end= True,\n",
        "          metric_for_best_model=\"accuracy\",\n",
        "          save_total_limit = 2,\n",
        "          seed = seed, \n",
        "      )\n",
        "\n",
        "      def model_init():\n",
        "          return RobertaForSequenceClassification.from_pretrained(\"roberta-base\", num_labels=nclasses)\n",
        "      trainer = Trainer(\n",
        "          model_init=model_init,               # the instantiated 🤗 Transformers model to be trained\n",
        "          args=training_args,                  # training arguments, defined above\n",
        "          train_dataset=train_dataset,         # training dataset\n",
        "          eval_dataset=dev_dataset,            # evaluation dataset\n",
        "          compute_metrics=compute_metrics,     # compute_metrics\n",
        "          )\n",
        "\n",
        "      trainer.train()\n",
        "      predictions = trainer.predict(test_dataset)\n",
        "      preds = np.argmax(predictions.predictions, axis=-1)\n",
        "\n",
        "      t1.append(top_k_accuracy(1, predictions.predictions, test_dataset.labels))\n",
        "\n",
        "    results[train_size] = t1\n",
        "  return results\n",
        "\n",
        "outputs = {}\n",
        "for task in tasks:\n",
        "  outputs[task] = compute_task(task)\n",
        "\n",
        "np.save( directory + \"figure_1_results.npy\", outputs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dL_Juhe54ttk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "20f9bf6d-dbcc-4d26-ae0d-22cf0327f1e2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "44-Topic Classification\n",
            "200 0.3938396176314392 0.014272184414012389\n",
            "300 0.4302708443972385 0.018305733848378577\n",
            "400 0.44503451938396166 0.011139764717686148\n",
            "8-Topic Classification\n",
            "200 0.5348911311736589 0.018559724542714137\n",
            "300 0.5472118959107807 0.015179984895792472\n",
            "400 0.5595326606479023 0.016832743779634632\n"
          ]
        }
      ],
      "source": [
        "for task in tasks:\n",
        "  print(task + \"-Topic Classification\")\n",
        "  results = outputs[task]\n",
        "  for train_size in results:\n",
        "    print(train_size, np.mean(results[train_size]), np.std(results[train_size]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nCt3Y9uBnikF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "171affca-943b-4923-9b8e-02bd8a717e4c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The program took 245.0 minutes in total.\n"
          ]
        }
      ],
      "source": [
        "end = time.time()\n",
        "print(f\"The program took {(end - start) // 60} minutes in total.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CdgJ4izEeLrY"
      },
      "outputs": [],
      "source": [
        "from google.colab import runtime\n",
        "runtime.unassign()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "040a69ae25d847d68fb4b6d37a3c1f7f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "099cc60e0e2340ac881c6beb35594b64": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "17ce202f23f7434ebafcdc77cf6d4e00": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "187b2c658c754f25be271e0e27774149": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2016118b6be84040af0d2c0586c43d14": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dcfc958e8b4146a3a41708180c72cee5",
            "max": 501200538,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e64c1c8790f549eea818bdb50a01feb6",
            "value": 501200538
          }
        },
        "238665a8a4cc45aa9b2d3d85380b5d4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_acd03816260e4583bbaf11c1fcd56982",
              "IPY_MODEL_2016118b6be84040af0d2c0586c43d14",
              "IPY_MODEL_417c0e82d111436fac1232d4a7850d64"
            ],
            "layout": "IPY_MODEL_c36f73c489cc430aa631af2f1c010d59"
          }
        },
        "23a003f440b545359b8010e9a7083e03": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "25ff7e39e5ec488f8bbad2168fba4a44": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2999cd51404e4bd8b40fa2c45b381bb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2a9f078224974a1eaf9d16c93ba627fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2bf51e9622ed462d8b55a45f12f2e915": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3561d189a4e461792d395f933521ddb",
            "placeholder": "​",
            "style": "IPY_MODEL_187b2c658c754f25be271e0e27774149",
            "value": " 899k/899k [00:00&lt;00:00, 1.64MB/s]"
          }
        },
        "2d1bc5b4baea44da8b83e9f9ff24be82": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3652b1df191d4326814059acadbf24c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a389a343a98b47669bde848a34adfab6",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_25ff7e39e5ec488f8bbad2168fba4a44",
            "value": 898823
          }
        },
        "389fb966970a45c68d58d76a927834ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc46a710d3e14349adcad4b1c3d74891",
            "max": 481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_099cc60e0e2340ac881c6beb35594b64",
            "value": 481
          }
        },
        "3cae7a02f919431a9e74509321c27783": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_040a69ae25d847d68fb4b6d37a3c1f7f",
            "max": 1355863,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_51c190453a6e49f8ae3d5a479f949ef6",
            "value": 1355863
          }
        },
        "3cba08ef6faa41e89534c785cd157259": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "417c0e82d111436fac1232d4a7850d64": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e23ecf61ef3a4f71ad51da5ae972d7a4",
            "placeholder": "​",
            "style": "IPY_MODEL_ba046666f06e4b61a5ea392314078d3e",
            "value": " 501M/501M [00:08&lt;00:00, 57.8MB/s]"
          }
        },
        "428a0c9f254744f18865480449e440da": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "45534ee3df6e483287993565172cdca0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2d1bc5b4baea44da8b83e9f9ff24be82",
            "placeholder": "​",
            "style": "IPY_MODEL_428a0c9f254744f18865480449e440da",
            "value": "Downloading: 100%"
          }
        },
        "4693a236332c420e8b0d847a086c6a99": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a6cd193e4574f3fbddf0efb04e4a933": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "51c190453a6e49f8ae3d5a479f949ef6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "59c21d08241f4546b1458534d2c65137": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ab058186b5a47a08a4d2eb3d4ddad40",
            "placeholder": "​",
            "style": "IPY_MODEL_23a003f440b545359b8010e9a7083e03",
            "value": " 481/481 [00:00&lt;00:00, 18.4kB/s]"
          }
        },
        "5ab058186b5a47a08a4d2eb3d4ddad40": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60896d137fae47f0bac72919b9735ef0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68ad546783a74e96830270f34303e52a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e1301421a704289a6bb7394ad3d1bdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_adb090506cbe47b6b39d4cb3888f87f2",
              "IPY_MODEL_ed618e494bdf48a3af03dcb604316003",
              "IPY_MODEL_b4b862be7f244e79a276f0c79d1b848a"
            ],
            "layout": "IPY_MODEL_60896d137fae47f0bac72919b9735ef0"
          }
        },
        "6e4a736d0dda4c0ba86ed4adce30bf06": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6f2bc607548944e79bea5dfd360c5d59": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6fbde38980d84dd281fdb780254018aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8dfb8ee3173841f69dc9cbb5507504e8",
            "placeholder": "​",
            "style": "IPY_MODEL_2a9f078224974a1eaf9d16c93ba627fe",
            "value": "Downloading: 100%"
          }
        },
        "7145848d480d4740a448dbba969454b1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8dfb8ee3173841f69dc9cbb5507504e8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ece54a75604436f9a36b5ccdeca7d11": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e85e18974a55458fafe8cea394dda2d1",
              "IPY_MODEL_3652b1df191d4326814059acadbf24c6",
              "IPY_MODEL_2bf51e9622ed462d8b55a45f12f2e915"
            ],
            "layout": "IPY_MODEL_d3c5bf2b63f143eda03ebbacf1042c4d"
          }
        },
        "9e0d568b2f594f3b9dfb6641797716a9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a389a343a98b47669bde848a34adfab6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acd03816260e4583bbaf11c1fcd56982": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e0d568b2f594f3b9dfb6641797716a9",
            "placeholder": "​",
            "style": "IPY_MODEL_f4b87dde40474fb792832cb12deea15f",
            "value": "Downloading: 100%"
          }
        },
        "adb090506cbe47b6b39d4cb3888f87f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68ad546783a74e96830270f34303e52a",
            "placeholder": "​",
            "style": "IPY_MODEL_4a6cd193e4574f3fbddf0efb04e4a933",
            "value": "Downloading builder script: "
          }
        },
        "aee36f2e6cc84db2b8c587c2cf07094e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b457c96412c94a4399e0a9bfc60e135f",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6e4a736d0dda4c0ba86ed4adce30bf06",
            "value": 456318
          }
        },
        "b308c42532cb4491ba78546a312b347e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_45534ee3df6e483287993565172cdca0",
              "IPY_MODEL_aee36f2e6cc84db2b8c587c2cf07094e",
              "IPY_MODEL_dcb07a89a02e41beb452fe3562b8aa31"
            ],
            "layout": "IPY_MODEL_4693a236332c420e8b0d847a086c6a99"
          }
        },
        "b457c96412c94a4399e0a9bfc60e135f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4b862be7f244e79a276f0c79d1b848a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_17ce202f23f7434ebafcdc77cf6d4e00",
            "placeholder": "​",
            "style": "IPY_MODEL_b560b6682f6442fc8e056587a96047bb",
            "value": " 4.21k/? [00:00&lt;00:00, 174kB/s]"
          }
        },
        "b560b6682f6442fc8e056587a96047bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b7716b58a6054808967e269832b8be96": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba046666f06e4b61a5ea392314078d3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bdf3a830b2824792b3cbb4731e68fdf1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e3965d9988284cae8e4c5a31e315fdd4",
            "placeholder": "​",
            "style": "IPY_MODEL_2999cd51404e4bd8b40fa2c45b381bb8",
            "value": "Downloading: 100%"
          }
        },
        "c3561d189a4e461792d395f933521ddb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c36f73c489cc430aa631af2f1c010d59": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c99f55fd2ff246e19e75ea8a3eea8ea2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbf68140c81c42cca33439cd062d4385": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ce4f4f8a574a4c8c86c104b11cbf23ce": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce50c195a786425fa88813c88810ffba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bdf3a830b2824792b3cbb4731e68fdf1",
              "IPY_MODEL_389fb966970a45c68d58d76a927834ee",
              "IPY_MODEL_59c21d08241f4546b1458534d2c65137"
            ],
            "layout": "IPY_MODEL_e051bd21f5a942139bfd9fd788720c6d"
          }
        },
        "d3c5bf2b63f143eda03ebbacf1042c4d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d73455019c34462b901793a560df11d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d9c741d200cf4c89b8fe66055c570a17": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc46a710d3e14349adcad4b1c3d74891": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dcb07a89a02e41beb452fe3562b8aa31": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c99f55fd2ff246e19e75ea8a3eea8ea2",
            "placeholder": "​",
            "style": "IPY_MODEL_3cba08ef6faa41e89534c785cd157259",
            "value": " 456k/456k [00:00&lt;00:00, 1.71MB/s]"
          }
        },
        "dcfc958e8b4146a3a41708180c72cee5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e051bd21f5a942139bfd9fd788720c6d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e23ecf61ef3a4f71ad51da5ae972d7a4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3965d9988284cae8e4c5a31e315fdd4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e64c1c8790f549eea818bdb50a01feb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e85e18974a55458fafe8cea394dda2d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7145848d480d4740a448dbba969454b1",
            "placeholder": "​",
            "style": "IPY_MODEL_cbf68140c81c42cca33439cd062d4385",
            "value": "Downloading: 100%"
          }
        },
        "ebc04124b24c4647af5a43e6055c1363": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6fbde38980d84dd281fdb780254018aa",
              "IPY_MODEL_3cae7a02f919431a9e74509321c27783",
              "IPY_MODEL_fdbf832dc70d4a64be71dfcbfd9ce0ee"
            ],
            "layout": "IPY_MODEL_ce4f4f8a574a4c8c86c104b11cbf23ce"
          }
        },
        "ed618e494bdf48a3af03dcb604316003": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d9c741d200cf4c89b8fe66055c570a17",
            "max": 1652,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d73455019c34462b901793a560df11d7",
            "value": 1652
          }
        },
        "f4b87dde40474fb792832cb12deea15f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fdbf832dc70d4a64be71dfcbfd9ce0ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f2bc607548944e79bea5dfd360c5d59",
            "placeholder": "​",
            "style": "IPY_MODEL_b7716b58a6054808967e269832b8be96",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 1.76MB/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}